<!DOCTYPE html><html lang="zh-CN" data-theme="light"><head><meta charset="UTF-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0,viewport-fit=cover"><title>爬虫二：动态网页的爬取 | 有求必应</title><meta name="author" content="Mr.wu"><meta name="copyright" content="Mr.wu"><meta name="format-detection" content="telephone=no"><meta name="theme-color" content="#ffffff"><meta name="description" content="动态网页动态网页的数据不会出现在页面的源代码中比如说马蜂窝网的自由行页面（https:&#x2F;&#x2F;www.mafengwo.cn&#x2F;sales&#x2F;0--M10030P%E6%B5%B7%E5%8D%97-0-0-0-0-0.html）  页面    其中活动列表的页面源码    渲染后页面   可以发现渲染前数据是没有的，如果我们像爬取静态网页一样爬取，那么只能得到未渲染的html结构 爬取方法一（selen">
<meta property="og:type" content="article">
<meta property="og:title" content="爬虫二：动态网页的爬取">
<meta property="og:url" content="https://comortgfytry.github.io/pages/%E7%88%AC%E8%99%AB/%E7%88%AC%E8%99%AB%E4%BA%8C%EF%BC%9A%E5%8A%A8%E6%80%81%E7%BD%91%E9%A1%B5%E7%9A%84%E7%88%AC%E5%8F%96.html">
<meta property="og:site_name" content="有求必应">
<meta property="og:description" content="动态网页动态网页的数据不会出现在页面的源代码中比如说马蜂窝网的自由行页面（https:&#x2F;&#x2F;www.mafengwo.cn&#x2F;sales&#x2F;0--M10030P%E6%B5%B7%E5%8D%97-0-0-0-0-0.html）  页面    其中活动列表的页面源码    渲染后页面   可以发现渲染前数据是没有的，如果我们像爬取静态网页一样爬取，那么只能得到未渲染的html结构 爬取方法一（selen">
<meta property="og:locale" content="zh_CN">
<meta property="og:image" content="https://comortgfytry.github.io/img/avatar.jpg">
<meta property="article:published_time" content="2023-07-22T11:23:34.000Z">
<meta property="article:modified_time" content="2023-07-24T02:23:05.365Z">
<meta property="article:author" content="Mr.wu">
<meta property="article:tag" content="Python">
<meta property="article:tag" content="爬虫">
<meta name="twitter:card" content="summary">
<meta name="twitter:image" content="https://comortgfytry.github.io/img/avatar.jpg"><link rel="shortcut icon" href="/img/favicon.png"><link rel="canonical" href="https://comortgfytry.github.io/pages/%E7%88%AC%E8%99%AB/%E7%88%AC%E8%99%AB%E4%BA%8C%EF%BC%9A%E5%8A%A8%E6%80%81%E7%BD%91%E9%A1%B5%E7%9A%84%E7%88%AC%E5%8F%96.html"><link rel="preconnect" href="//cdn.jsdelivr.net"/><link rel="preconnect" href="//busuanzi.ibruce.info"/><link rel="stylesheet" href="/css/index.css"><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@fortawesome/fontawesome-free/css/all.min.css" media="print" onload="this.media='all'"><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@fancyapps/ui/dist/fancybox/fancybox.min.css" media="print" onload="this.media='all'"><script>const GLOBAL_CONFIG = {
  root: '/',
  algolia: undefined,
  localSearch: {"path":"/search.xml","preload":true,"top_n_per_article":1,"unescape":false,"languages":{"hits_empty":"找不到您查询的内容：${query}","hits_stats":"共找到 ${hits} 篇文章"}},
  translate: undefined,
  noticeOutdate: undefined,
  highlight: {"plugin":"highlighjs","highlightCopy":true,"highlightLang":true,"highlightHeightLimit":300},
  copy: {
    success: '复制成功',
    error: '复制错误',
    noSupport: '浏览器不支持'
  },
  relativeDate: {
    homepage: false,
    post: false
  },
  runtime: '',
  dateSuffix: {
    just: '刚刚',
    min: '分钟前',
    hour: '小时前',
    day: '天前',
    month: '个月前'
  },
  copyright: {"limitCount":200,"languages":{"author":"作者: Mr.wu","link":"链接: ","source":"来源: 有求必应","info":"著作权归作者所有。商业转载请联系作者获得授权，非商业转载请注明出处。"}},
  lightbox: 'fancybox',
  Snackbar: undefined,
  source: {
    justifiedGallery: {
      js: 'https://cdn.jsdelivr.net/npm/flickr-justified-gallery/dist/fjGallery.min.js',
      css: 'https://cdn.jsdelivr.net/npm/flickr-justified-gallery/dist/fjGallery.min.css'
    }
  },
  isPhotoFigcaption: false,
  islazyload: true,
  isAnchor: false,
  percent: {
    toc: true,
    rightside: false,
  },
  autoDarkmode: false
}</script><script id="config-diff">var GLOBAL_CONFIG_SITE = {
  title: '爬虫二：动态网页的爬取',
  isPost: true,
  isHome: false,
  isHighlightShrink: undefined,
  isToc: true,
  postUpdate: '2023-07-24 10:23:05'
}</script><noscript><style type="text/css">
  #nav {
    opacity: 1
  }
  .justified-gallery img {
    opacity: 1
  }

  #recent-posts time,
  #post-meta time {
    display: inline !important
  }
</style></noscript><script>(win=>{
    win.saveToLocal = {
      set: function setWithExpiry(key, value, ttl) {
        if (ttl === 0) return
        const now = new Date()
        const expiryDay = ttl * 86400000
        const item = {
          value: value,
          expiry: now.getTime() + expiryDay,
        }
        localStorage.setItem(key, JSON.stringify(item))
      },

      get: function getWithExpiry(key) {
        const itemStr = localStorage.getItem(key)

        if (!itemStr) {
          return undefined
        }
        const item = JSON.parse(itemStr)
        const now = new Date()

        if (now.getTime() > item.expiry) {
          localStorage.removeItem(key)
          return undefined
        }
        return item.value
      }
    }
  
    win.getScript = url => new Promise((resolve, reject) => {
      const script = document.createElement('script')
      script.src = url
      script.async = true
      script.onerror = reject
      script.onload = script.onreadystatechange = function() {
        const loadState = this.readyState
        if (loadState && loadState !== 'loaded' && loadState !== 'complete') return
        script.onload = script.onreadystatechange = null
        resolve()
      }
      document.head.appendChild(script)
    })
  
    win.getCSS = (url,id = false) => new Promise((resolve, reject) => {
      const link = document.createElement('link')
      link.rel = 'stylesheet'
      link.href = url
      if (id) link.id = id
      link.onerror = reject
      link.onload = link.onreadystatechange = function() {
        const loadState = this.readyState
        if (loadState && loadState !== 'loaded' && loadState !== 'complete') return
        link.onload = link.onreadystatechange = null
        resolve()
      }
      document.head.appendChild(link)
    })
  
      win.activateDarkMode = function () {
        document.documentElement.setAttribute('data-theme', 'dark')
        if (document.querySelector('meta[name="theme-color"]') !== null) {
          document.querySelector('meta[name="theme-color"]').setAttribute('content', '#0d0d0d')
        }
      }
      win.activateLightMode = function () {
        document.documentElement.setAttribute('data-theme', 'light')
        if (document.querySelector('meta[name="theme-color"]') !== null) {
          document.querySelector('meta[name="theme-color"]').setAttribute('content', '#ffffff')
        }
      }
      const t = saveToLocal.get('theme')
    
          if (t === 'dark') activateDarkMode()
          else if (t === 'light') activateLightMode()
        
      const asideStatus = saveToLocal.get('aside-status')
      if (asideStatus !== undefined) {
        if (asideStatus === 'hide') {
          document.documentElement.classList.add('hide-aside')
        } else {
          document.documentElement.classList.remove('hide-aside')
        }
      }
    
    const detectApple = () => {
      if(/iPad|iPhone|iPod|Macintosh/.test(navigator.userAgent)){
        document.documentElement.classList.add('apple')
      }
    }
    detectApple()
    })(window)</script><link rel="stylesheet" href="/css/mycss.css"><style type="text/css"> #universe {  display: block; position: fixed; margin: 0; padding: 0; border: 0; outline: 0; left: 0; top: 0; width: 100%; height: 100%; pointer-events: none; z-index: 1; } </style><meta name="generator" content="Hexo 6.3.0"></head><body><div id="web_bg"></div><div id="sidebar"><div id="menu-mask"></div><div id="sidebar-menus"><div class="avatar-img is-center"><img src= "" data-lazy-src="/img/avatar.jpg" onerror="onerror=null;src='/img/friend_404.gif'" alt="avatar"/></div><div class="sidebar-site-data site-data is-center"><a href="/archives/"><div class="headline">文章</div><div class="length-num">63</div></a><a href="/tags/"><div class="headline">标签</div><div class="length-num">32</div></a><a href="/categories/"><div class="headline">分类</div><div class="length-num">25</div></a></div><hr class="custom-hr"/><div class="menus_items"><div class="menus_item"><a class="site-page" href="/"><i class="fa-fw fas fa-home"></i><span> 主页</span></a></div><div class="menus_item"><a class="site-page group" href="javascript:void(0);"><i class="fa-fw fa fa-bookmark"></i><span> 博文</span><i class="fas fa-chevron-down"></i></a><ul class="menus_item_child"><li><a class="site-page child" href="/categories/"><i class="fa-fw fa fa-archive"></i><span> 分类</span></a></li><li><a class="site-page child" href="/tags/"><i class="fa-fw fa fa-tags"></i><span> 标签</span></a></li><li><a class="site-page child" href="/archives/"><i class="fa-fw fa fa-folder-open"></i><span> 归档</span></a></li></ul></div><div class="menus_item"><a class="site-page" href="/comment/"><i class="fa-fw fa fa-paper-plane"></i><span> 留言板</span></a></div><div class="menus_item"><a class="site-page" href="/about/"><i class="fa-fw fas fa-user"></i><span> 关于我</span></a></div></div></div></div><div class="post" id="body-wrap"><header class="post-bg" id="page-header" style="background-image: url('https://api.aqcoder.cn/random')"><nav id="nav"><span id="blog-info"><a href="/" title="有求必应"><span class="site-name">有求必应</span></a></span><div id="menus"><div id="search-button"><a class="site-page social-icon search" href="javascript:void(0);"><i class="fas fa-search fa-fw"></i><span> 搜索</span></a></div><div class="menus_items"><div class="menus_item"><a class="site-page" href="/"><i class="fa-fw fas fa-home"></i><span> 主页</span></a></div><div class="menus_item"><a class="site-page group" href="javascript:void(0);"><i class="fa-fw fa fa-bookmark"></i><span> 博文</span><i class="fas fa-chevron-down"></i></a><ul class="menus_item_child"><li><a class="site-page child" href="/categories/"><i class="fa-fw fa fa-archive"></i><span> 分类</span></a></li><li><a class="site-page child" href="/tags/"><i class="fa-fw fa fa-tags"></i><span> 标签</span></a></li><li><a class="site-page child" href="/archives/"><i class="fa-fw fa fa-folder-open"></i><span> 归档</span></a></li></ul></div><div class="menus_item"><a class="site-page" href="/comment/"><i class="fa-fw fa fa-paper-plane"></i><span> 留言板</span></a></div><div class="menus_item"><a class="site-page" href="/about/"><i class="fa-fw fas fa-user"></i><span> 关于我</span></a></div></div><div id="toggle-menu"><a class="site-page" href="javascript:void(0);"><i class="fas fa-bars fa-fw"></i></a></div></div></nav><div id="post-info"><h1 class="post-title">爬虫二：动态网页的爬取</h1><div id="post-meta"><div class="meta-firstline"><span class="post-meta-date"><i class="far fa-calendar-alt fa-fw post-meta-icon"></i><span class="post-meta-label">发表于</span><time class="post-meta-date-created" datetime="2023-07-22T11:23:34.000Z" title="发表于 2023-07-22 19:23:34">2023-07-22</time><span class="post-meta-separator">|</span><i class="fas fa-history fa-fw post-meta-icon"></i><span class="post-meta-label">更新于</span><time class="post-meta-date-updated" datetime="2023-07-24T02:23:05.365Z" title="更新于 2023-07-24 10:23:05">2023-07-24</time></span><span class="post-meta-categories"><span class="post-meta-separator">|</span><i class="fas fa-inbox fa-fw post-meta-icon"></i><a class="post-meta-categories" href="/categories/%E7%88%AC%E8%99%AB/">爬虫</a></span></div><div class="meta-secondline"><span class="post-meta-separator">|</span><span class="post-meta-wordcount"><i class="far fa-file-word fa-fw post-meta-icon"></i><span class="post-meta-label">字数总计:</span><span class="word-count">1.3k</span><span class="post-meta-separator">|</span><i class="far fa-clock fa-fw post-meta-icon"></i><span class="post-meta-label">阅读时长:</span><span>5分钟</span></span><span class="post-meta-separator">|</span><span class="post-meta-pv-cv" id="" data-flag-title="爬虫二：动态网页的爬取"><i class="far fa-eye fa-fw post-meta-icon"></i><span class="post-meta-label">阅读量:</span><span id="busuanzi_value_page_pv"><i class="fa-solid fa-spinner fa-spin"></i></span></span></div></div></div></header><main class="layout" id="content-inner"><div id="post"><article class="post-content" id="article-container"><h1 id="动态网页"><a href="#动态网页" class="headerlink" title="动态网页"></a>动态网页</h1><p>动态网页的数据不会出现在页面的源代码中<br>比如说马蜂窝网的自由行页面（<a target="_blank" rel="noopener" href="https://www.mafengwo.cn/sales/0--M10030P%E6%B5%B7%E5%8D%97-0-0-0-0-0.html%EF%BC%89">https://www.mafengwo.cn/sales/0--M10030P%E6%B5%B7%E5%8D%97-0-0-0-0-0.html）</a></p>
<ul>
<li>页面<br><img src= "" data-lazy-src="/pages/%E7%88%AC%E8%99%AB/%E7%88%AC%E8%99%AB%E4%BA%8C%EF%BC%9A%E5%8A%A8%E6%80%81%E7%BD%91%E9%A1%B5%E7%9A%84%E7%88%AC%E5%8F%96/image.png" alt="image">   </li>
<li>其中活动列表的页面源码</li>
</ul>
<p><img src= "" data-lazy-src="/pages/%E7%88%AC%E8%99%AB/%E7%88%AC%E8%99%AB%E4%BA%8C%EF%BC%9A%E5%8A%A8%E6%80%81%E7%BD%91%E9%A1%B5%E7%9A%84%E7%88%AC%E5%8F%96/image1.png" alt="image1"></p>
<ul>
<li>渲染后页面</li>
</ul>
<p><img src= "" data-lazy-src="/pages/%E7%88%AC%E8%99%AB/%E7%88%AC%E8%99%AB%E4%BA%8C%EF%BC%9A%E5%8A%A8%E6%80%81%E7%BD%91%E9%A1%B5%E7%9A%84%E7%88%AC%E5%8F%96/image2.png" alt="iamge2"></p>
<p>可以发现渲染前数据是没有的，如果我们像爬取静态网页一样爬取，那么只能得到未渲染的html结构</p>
<h1 id="爬取方法一（selenium模拟浏览器）"><a href="#爬取方法一（selenium模拟浏览器）" class="headerlink" title="爬取方法一（selenium模拟浏览器）"></a>爬取方法一（selenium模拟浏览器）</h1><h2 id="配置环境"><a href="#配置环境" class="headerlink" title="配置环境"></a>配置环境</h2><ul>
<li>安装selenium</li>
</ul>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">pip install selenium</span><br></pre></td></tr></table></figure>
<ul>
<li>安装谷歌浏览器</li>
<li>安装chromedriver</li>
</ul>
<blockquote>
<p>注意查看自己谷歌浏览器版本，然后下载对应（大版本相同即可）的chromedriver<br>链接：<a target="_blank" rel="noopener" href="https://registry.npmmirror.com/binary.html?path=chromedriver/">https://registry.npmmirror.com/binary.html?path=chromedriver/</a><br>解压后将其放到Python安装路径下  </p>
</blockquote>
<h2 id="实现（爬取动态页面内容-虚拟点击翻页）"><a href="#实现（爬取动态页面内容-虚拟点击翻页）" class="headerlink" title="实现（爬取动态页面内容 + 虚拟点击翻页）"></a>实现（爬取动态页面内容 + 虚拟点击翻页）</h2><ul>
<li>目标：爬取三亚自由行（<a target="_blank" rel="noopener" href="https://www.mafengwo.cn/sales/0--M10030P%E6%B5%B7%E5%8D%97-0-0-0-0-0.html%EF%BC%89%E7%9A%84%E7%AC%AC%E5%89%8D6%E9%A1%B5%E6%B4%BB%E5%8A%A8">https://www.mafengwo.cn/sales/0--M10030P%E6%B5%B7%E5%8D%97-0-0-0-0-0.html）的第前6页活动</a></li>
</ul>
<h3 id="获取渲染后的页面"><a href="#获取渲染后的页面" class="headerlink" title="获取渲染后的页面"></a>获取渲染后的页面</h3><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">browser = webdriver.Chrome()</span><br><span class="line">browser.get(url1)</span><br></pre></td></tr></table></figure>
<h3 id="定位爬取内容的方法"><a href="#定位爬取内容的方法" class="headerlink" title="定位爬取内容的方法"></a>定位爬取内容的方法</h3><h4 id="F12打开页面元素如图获取xpath"><a href="#F12打开页面元素如图获取xpath" class="headerlink" title="F12打开页面元素如图获取xpath"></a>F12打开页面元素如图获取xpath</h4><p><img src= "" data-lazy-src="/pages/%E7%88%AC%E8%99%AB/%E7%88%AC%E8%99%AB%E4%BA%8C%EF%BC%9A%E5%8A%A8%E6%80%81%E7%BD%91%E9%A1%B5%E7%9A%84%E7%88%AC%E5%8F%96/image3.png" alt="image3"></p>
<h4 id="获取元素中想要的部分"><a href="#获取元素中想要的部分" class="headerlink" title="获取元素中想要的部分"></a>获取元素中想要的部分</h4><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">element = browser.find_element_by_xpath(<span class="string">&#x27;xpath&#x27;</span>)</span><br></pre></td></tr></table></figure>

<h4 id="进一步获取"><a href="#进一步获取" class="headerlink" title="进一步获取"></a>进一步获取</h4><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">element.get_property(属性) <span class="comment"># 获取属性的内容</span></span><br></pre></td></tr></table></figure>

<p>使用：  </p>
<figure class="highlight html"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">element:<span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span> = <span class="string">&quot;/sales/21023.html&quot;</span>&gt;</span><span class="tag">&lt;/<span class="name">a</span>&gt;</span></span><br></pre></td></tr></table></figure>

<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">element.get_property(<span class="string">&#x27;href&#x27;</span>) <span class="comment"># 获取了&lt;a&gt;标签中的链接&quot;/sales/21023.html&quot;</span></span><br></pre></td></tr></table></figure>
<p> <br>————————————————————————– <br> </p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">element.text <span class="comment"># 获取标签的内容</span></span><br></pre></td></tr></table></figure>

<p>使用：</p>
<figure class="highlight html"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">element:<span class="tag">&lt;<span class="name">a</span>&gt;</span>hello<span class="tag">&lt;/<span class="name">a</span>&gt;</span></span><br></pre></td></tr></table></figure>

<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">element.text <span class="comment"># 得到hello</span></span><br></pre></td></tr></table></figure>

<p> <br>————————————————————————– <br> </p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">element.find_element_by_link_text(标签内容) <span class="comment">#根据标签内容获取标签（反过来）</span></span><br></pre></td></tr></table></figure>

<p>使用:</p>
<figure class="highlight html"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">element:<span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">&quot;javascript:void(0);&quot;</span> <span class="attr">class</span>=<span class="string">&quot;pi pg-next&quot;</span>&gt;</span>Next&gt;&gt;<span class="tag">&lt;/<span class="name">a</span>&gt;</span></span><br></pre></td></tr></table></figure>

<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">next_page = element.find_element_by_link_text(<span class="string">&#x27;Next&gt;&gt;&#x27;</span>) <span class="comment"># 得到标签，可以模拟其功能。这里是一个跳转到下一页的标签，所以可以调用click()模拟跳转</span></span><br><span class="line">next_page.click()</span><br></pre></td></tr></table></figure>

<p> <br>————————————————————————– <br> </p>
 <figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">element.find_element_by_class_name(<span class="keyword">class</span>属性内容)</span><br></pre></td></tr></table></figure>

<p> 经常使用</p>
<p>当然还有许多方法，这里不多叙述</p>
<h3 id="完整代码"><a href="#完整代码" class="headerlink" title="完整代码"></a>完整代码</h3><p>输入某城市自由行页面第一页链接即可爬取6页活动（example：’<a target="_blank" rel="noopener" href="https://www.mafengwo.cn/sales/0-10320-M14674-0-0-0-0-0.html?group=1&seid=23F35AC5-65E0-4048-8F1B-4677F6B87E7A%27%EF%BC%89">https://www.mafengwo.cn/sales/0-10320-M14674-0-0-0-0-0.html?group=1&amp;seid=23F35AC5-65E0-4048-8F1B-4677F6B87E7A&#39;）</a></p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">from</span> selenium <span class="keyword">import</span> webdriver</span><br><span class="line"><span class="keyword">from</span> bs4 <span class="keyword">import</span> BeautifulSoup</span><br><span class="line"><span class="keyword">from</span> selenium.webdriver.support.ui <span class="keyword">import</span> WebDriverWait</span><br><span class="line"><span class="keyword">from</span> selenium.webdriver.common.by <span class="keyword">import</span> By</span><br><span class="line"><span class="keyword">import</span> requests</span><br><span class="line"><span class="keyword">from</span> selenium.webdriver.support <span class="keyword">import</span> expected_conditions</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">headers = &#123;</span><br><span class="line">    <span class="string">&quot;User-Agent&quot;</span>:<span class="string">&quot;Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36 Edg/114.0.1823.67&quot;</span></span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line">url = <span class="built_in">input</span>(<span class="string">&#x27;某城市自由行页面链接：&#x27;</span>)</span><br><span class="line">browser = webdriver.Chrome()</span><br><span class="line">browser.get(url)</span><br><span class="line">locator = (By.LINK_TEXT, <span class="string">&#x27;Next&gt;&gt;&#x27;</span>)</span><br><span class="line"></span><br><span class="line">WebDriverWait(browser, <span class="number">16</span>).until(expected_conditions.presence_of_element_located(locator))</span><br><span class="line"></span><br><span class="line">htmls = []</span><br><span class="line"></span><br><span class="line">f = <span class="built_in">open</span>(<span class="string">&quot;activities.txt&quot;</span>, <span class="string">&quot;w&quot;</span>, encoding=<span class="string">&#x27;UTF-8&#x27;</span>)</span><br><span class="line"><span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">1</span>, <span class="number">5</span>):</span><br><span class="line">    <span class="comment"># 获取每个活动的网页链接</span></span><br><span class="line">    <span class="keyword">for</span> aTag <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">1</span>, <span class="number">21</span>):</span><br><span class="line">        element = browser.find_element_by_xpath(<span class="string">&#x27;/html/body/div[2]/div[2]/div[4]/a[&#x27;</span> + <span class="built_in">str</span>(aTag) + <span class="string">&#x27;]&#x27;</span>)</span><br><span class="line">        htmls.append(element.get_property(<span class="string">&#x27;href&#x27;</span>))</span><br><span class="line"></span><br><span class="line">    <span class="comment"># print(htmls)</span></span><br><span class="line">    <span class="comment"># 获取活动的html</span></span><br><span class="line">    <span class="comment"># https://www.mafengwo.cn/sales/2295490.html</span></span><br><span class="line"></span><br><span class="line">    <span class="keyword">for</span> url <span class="keyword">in</span> htmls:</span><br><span class="line">        response = requests.get(url, headers=headers);</span><br><span class="line">        html = response.text</span><br><span class="line">        soup = BeautifulSoup(html, <span class="string">&#x27;lxml&#x27;</span>)</span><br><span class="line">        tmp = soup.find(name=<span class="string">&#x27;div&#x27;</span>, class_=<span class="string">&#x27;sales-title&#x27;</span>)</span><br><span class="line">        <span class="keyword">if</span> tmp != <span class="literal">None</span>:</span><br><span class="line">            tmp = tmp.find(name=<span class="string">&#x27;h1&#x27;</span>)</span><br><span class="line">            <span class="keyword">if</span> tmp != <span class="literal">None</span>:</span><br><span class="line">                service = tmp.text</span><br><span class="line">            <span class="keyword">else</span>:</span><br><span class="line">                service = <span class="string">&#x27;***&#x27;</span></span><br><span class="line">        <span class="keyword">else</span>:</span><br><span class="line">            service = <span class="string">&#x27;***&#x27;</span></span><br><span class="line">        <span class="built_in">print</span>(service)</span><br><span class="line">        tmp = soup.find(name=<span class="string">&#x27;strong&#x27;</span>, class_=<span class="string">&#x27;js_dingzhi_price&#x27;</span>)</span><br><span class="line">        <span class="keyword">if</span> tmp != <span class="literal">None</span>:</span><br><span class="line">            price = tmp.text</span><br><span class="line">        <span class="keyword">else</span>:</span><br><span class="line">            price = <span class="string">&#x27;***&#x27;</span></span><br><span class="line">        <span class="built_in">print</span>(price)</span><br><span class="line">        tmp = soup.find(name=<span class="string">&#x27;span&#x27;</span>, class_=<span class="string">&#x27;soldout&#x27;</span>)</span><br><span class="line">        <span class="keyword">if</span> tmp != <span class="literal">None</span>:</span><br><span class="line">            sold = tmp.text</span><br><span class="line">        <span class="keyword">else</span>:</span><br><span class="line">            sold = <span class="string">&#x27;***&#x27;</span></span><br><span class="line"></span><br><span class="line">        activity = service + <span class="string">&#x27;,&#x27;</span> + price + <span class="string">&#x27;,&#x27;</span> + sold</span><br><span class="line">        f.write(activity)</span><br><span class="line">        f.write(<span class="string">&#x27;\n&#x27;</span>)</span><br><span class="line">    <span class="comment"># 实现虚拟点击</span></span><br><span class="line">    next_page = browser.find_element_by_link_text(<span class="string">&#x27;Next&gt;&gt;&#x27;</span>)</span><br><span class="line">    next_page.click()</span><br><span class="line">f.close()</span><br></pre></td></tr></table></figure>

<h1 id="爬取方法二（通过javsript逆向获取数据接口）"><a href="#爬取方法二（通过javsript逆向获取数据接口）" class="headerlink" title="爬取方法二（通过javsript逆向获取数据接口）"></a>爬取方法二（通过javsript逆向获取数据接口）</h1><p>基本方法：在需要爬取网页F12选择网络，选择XHR,点击名称，选择预览。依次查看预览，找到想要爬取的内容，再切换到标头，找到请求url（真实需要爬取的链接），用该链接爬取预览中的内容（大部分是json）<br>好处：比selenium效率更高</p>
<h2 id="爬取今日头条的头条热榜（https-www-toutiao-com-）"><a href="#爬取今日头条的头条热榜（https-www-toutiao-com-）" class="headerlink" title="爬取今日头条的头条热榜（https://www.toutiao.com/）"></a>爬取今日头条的头条热榜（<a target="_blank" rel="noopener" href="https://www.toutiao.com/%EF%BC%89">https://www.toutiao.com/）</a></h2><h3 id="分析"><a href="#分析" class="headerlink" title="分析"></a>分析</h3><p><img src= "" data-lazy-src="/pages/%E7%88%AC%E8%99%AB/%E7%88%AC%E8%99%AB%E4%BA%8C%EF%BC%9A%E5%8A%A8%E6%80%81%E7%BD%91%E9%A1%B5%E7%9A%84%E7%88%AC%E5%8F%96/image6.png" alt="image6"><br><img src= "" data-lazy-src="/pages/%E7%88%AC%E8%99%AB/%E7%88%AC%E8%99%AB%E4%BA%8C%EF%BC%9A%E5%8A%A8%E6%80%81%E7%BD%91%E9%A1%B5%E7%9A%84%E7%88%AC%E5%8F%96/image4.png" alt="image4"><br><img src= "" data-lazy-src="/pages/%E7%88%AC%E8%99%AB/%E7%88%AC%E8%99%AB%E4%BA%8C%EF%BC%9A%E5%8A%A8%E6%80%81%E7%BD%91%E9%A1%B5%E7%9A%84%E7%88%AC%E5%8F%96/image5.png" alt="image5"></p>
<p>可以发现头条热榜中内容是json格式</p>
<h3 id="实现"><a href="#实现" class="headerlink" title="实现"></a>实现</h3><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">import</span> requests</span><br><span class="line"></span><br><span class="line">url = <span class="string">&#x27;https://www.toutiao.com/hot-event/hot-board/?origin=toutiao_pc&amp;_signature=_02B4Z6wo00f01cm1l.wAAIDAPMp9YrQAl5HJkZNAABbSIRLm0FZII5tPd8u2PIsOqj1YucYIEdGWkcDPpbZOxzsho28lefSfMc0QiN1YHh9hAjgdADwz4lHocFqMV1YF3n8um5OwWNJj77DX16&#x27;</span></span><br><span class="line"></span><br><span class="line">headers = &#123;</span><br><span class="line">    <span class="string">&quot;User-Agent&quot;</span>:<span class="string">&quot;Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36 Edg/114.0.1823.67&quot;</span></span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line">response = requests.get(url, headers=headers)</span><br><span class="line">json = response.json()</span><br><span class="line"><span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">0</span>, <span class="built_in">len</span>(json[<span class="string">&#x27;data&#x27;</span>])):</span><br><span class="line">    <span class="built_in">print</span>(json[<span class="string">&#x27;data&#x27;</span>][i][<span class="string">&#x27;Title&#x27;</span>])</span><br></pre></td></tr></table></figure>

<h2 id="爬取马蜂窝网自由行列表（https-www-mafengwo-cn-sales-0-0-M10030-0-0-0-0-0-html）"><a href="#爬取马蜂窝网自由行列表（https-www-mafengwo-cn-sales-0-0-M10030-0-0-0-0-0-html）" class="headerlink" title="爬取马蜂窝网自由行列表（https://www.mafengwo.cn/sales/0-0-M10030-0-0-0-0-0.html）"></a>爬取马蜂窝网自由行列表（<a target="_blank" rel="noopener" href="https://www.mafengwo.cn/sales/0-0-M10030-0-0-0-0-0.html%EF%BC%89">https://www.mafengwo.cn/sales/0-0-M10030-0-0-0-0-0.html）</a></h2><h3 id="分析-1"><a href="#分析-1" class="headerlink" title="分析"></a>分析</h3><p><img src= "" data-lazy-src="/pages/%E7%88%AC%E8%99%AB/%E7%88%AC%E8%99%AB%E4%BA%8C%EF%BC%9A%E5%8A%A8%E6%80%81%E7%BD%91%E9%A1%B5%E7%9A%84%E7%88%AC%E5%8F%96/image7.png" alt="image7"><br><img src= "" data-lazy-src="/pages/%E7%88%AC%E8%99%AB/%E7%88%AC%E8%99%AB%E4%BA%8C%EF%BC%9A%E5%8A%A8%E6%80%81%E7%BD%91%E9%A1%B5%E7%9A%84%E7%88%AC%E5%8F%96/image8.png" alt="image8"><br><img src= "" data-lazy-src="/pages/%E7%88%AC%E8%99%AB/%E7%88%AC%E8%99%AB%E4%BA%8C%EF%BC%9A%E5%8A%A8%E6%80%81%E7%BD%91%E9%A1%B5%E7%9A%84%E7%88%AC%E5%8F%96/image9.png" alt="image9"></p>
<p>可以发现这个页面全部把数据存到了html中</p>
<h3 id="实现-1"><a href="#实现-1" class="headerlink" title="实现"></a>实现</h3><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">import</span> requests</span><br><span class="line"><span class="keyword">from</span> bs4 <span class="keyword">import</span> BeautifulSoup</span><br><span class="line"></span><br><span class="line">url = <span class="string">&#x27;https://www.mafengwo.cn/sales/ajax_2017.php?act=GetContentList&amp;s_dept_time%5B%5D=all&amp;price%5B%5D=all&amp;from=NaN&amp;kw=&amp;to=M10030&amp;salesType=NaN&amp;page=1&amp;group=1&amp;sort=smart&amp;sort_type=desc&amp;limit=20&#x27;</span></span><br><span class="line">headers = &#123;</span><br><span class="line">    <span class="string">&quot;User-Agent&quot;</span>: <span class="string">&quot;Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36 Edg/114.0.1823.67&quot;</span></span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line">response = requests.get(url, headers=headers)</span><br><span class="line">json = response.json()</span><br><span class="line">html = json[<span class="string">&#x27;html&#x27;</span>]</span><br><span class="line">soup = BeautifulSoup(html, <span class="string">&#x27;lxml&#x27;</span>)</span><br><span class="line"><span class="keyword">for</span> h3 <span class="keyword">in</span> soup.find_all(name=<span class="string">&#x27;h3&#x27;</span>):</span><br><span class="line">    <span class="built_in">print</span>(h3.text)</span><br><span class="line"></span><br></pre></td></tr></table></figure>
<p>此时如果需要翻页，可以发现每一页的链接发生变化，使用了page&#x3D;x来表示第x页。所以创建循环拼接链接就可以了</p>
</article><div class="post-copyright"><div class="post-copyright__author"><span class="post-copyright-meta">文章作者: </span><span class="post-copyright-info"><a href="https://comortgfytry.github.io">Mr.wu</a></span></div><div class="post-copyright__type"><span class="post-copyright-meta">文章链接: </span><span class="post-copyright-info"><a href="https://comortgfytry.github.io/pages/%E7%88%AC%E8%99%AB/%E7%88%AC%E8%99%AB%E4%BA%8C%EF%BC%9A%E5%8A%A8%E6%80%81%E7%BD%91%E9%A1%B5%E7%9A%84%E7%88%AC%E5%8F%96.html">https://comortgfytry.github.io/pages/%E7%88%AC%E8%99%AB/%E7%88%AC%E8%99%AB%E4%BA%8C%EF%BC%9A%E5%8A%A8%E6%80%81%E7%BD%91%E9%A1%B5%E7%9A%84%E7%88%AC%E5%8F%96.html</a></span></div><div class="post-copyright__notice"><span class="post-copyright-meta">版权声明: </span><span class="post-copyright-info">本博客所有文章除特别声明外，均采用 <a href="https://creativecommons.org/licenses/by-nc-sa/4.0/" target="_blank">CC BY-NC-SA 4.0</a> 许可协议。转载请注明来自 <a href="https://comortgfytry.github.io" target="_blank">有求必应</a>！</span></div></div><div class="tag_share"><div class="post-meta__tag-list"><a class="post-meta__tags" href="/tags/Python/">Python</a><a class="post-meta__tags" href="/tags/%E7%88%AC%E8%99%AB/">爬虫</a></div><div class="post_share"><div class="social-share" data-image="/img/avatar.jpg" data-sites="facebook,twitter,wechat,weibo,qq"></div><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/butterfly-extsrc/sharejs/dist/css/share.min.css" media="print" onload="this.media='all'"><script src="https://cdn.jsdelivr.net/npm/butterfly-extsrc/sharejs/dist/js/social-share.min.js" defer></script></div></div><nav class="pagination-post" id="pagination"><div class="prev-post pull-left"><a href="/pages/%E7%88%AC%E8%99%AB/%E7%88%AC%E8%99%AB%E4%B8%80%EF%BC%9A%E7%AE%80%E5%8D%95%E9%9D%99%E6%80%81%E7%BD%91%E9%A1%B5%E7%9A%84%E7%88%AC%E5%8F%96.html" title="爬虫一：简单静态网页的爬取"><div class="cover" style="background: var(--default-bg-color)"></div><div class="pagination-info"><div class="label">上一篇</div><div class="prev_info">爬虫一：简单静态网页的爬取</div></div></a></div><div class="next-post pull-right"><a href="/pages/flask%E6%A1%86%E6%9E%B6/flask%E5%85%A5%E9%97%A8%E4%BD%BF%E7%94%A8.html" title="flask入门使用"><div class="cover" style="background: var(--default-bg-color)"></div><div class="pagination-info"><div class="label">下一篇</div><div class="next_info">flask入门使用</div></div></a></div></nav><div class="relatedPosts"><div class="headline"><i class="fas fa-thumbs-up fa-fw"></i><span>相关推荐</span></div><div class="relatedPosts-list"><div><a href="/pages/%E7%88%AC%E8%99%AB/%E5%88%86%E5%B8%83%E5%BC%8F%E7%88%AC%E8%99%AB.html" title="分布式爬虫"><div class="cover" style="background: var(--default-bg-color)"></div><div class="content is-center"><div class="date"><i class="far fa-calendar-alt fa-fw"></i> 2023-07-19</div><div class="title">分布式爬虫</div></div></a></div><div><a href="/pages/%E7%88%AC%E8%99%AB/%E7%88%AC%E8%99%AB%E4%B8%80%EF%BC%9A%E7%AE%80%E5%8D%95%E9%9D%99%E6%80%81%E7%BD%91%E9%A1%B5%E7%9A%84%E7%88%AC%E5%8F%96.html" title="爬虫一：简单静态网页的爬取"><div class="cover" style="background: var(--default-bg-color)"></div><div class="content is-center"><div class="date"><i class="far fa-calendar-alt fa-fw"></i> 2023-07-21</div><div class="title">爬虫一：简单静态网页的爬取</div></div></a></div><div><a href="/pages/%E7%88%AC%E8%99%AB/%E7%88%AC%E8%99%AB%E5%B8%B8%E8%A7%81%E9%97%AE%E9%A2%98%E6%80%BB%E7%BB%93.html" title="爬虫常见问题总结"><div class="cover" style="background: var(--default-bg-color)"></div><div class="content is-center"><div class="date"><i class="far fa-calendar-alt fa-fw"></i> 2023-07-25</div><div class="title">爬虫常见问题总结</div></div></a></div><div><a href="/pages/flask%E6%A1%86%E6%9E%B6/flask%E5%85%A5%E9%97%A8%E4%BD%BF%E7%94%A8.html" title="flask入门使用"><div class="cover" style="background: var(--default-bg-color)"></div><div class="content is-center"><div class="date"><i class="far fa-calendar-alt fa-fw"></i> 2023-07-24</div><div class="title">flask入门使用</div></div></a></div></div></div></div><div class="aside-content" id="aside-content"><div class="card-widget card-info"><div class="is-center"><div class="avatar-img"><img src= "" data-lazy-src="/img/avatar.jpg" onerror="this.onerror=null;this.src='/img/friend_404.gif'" alt="avatar"/></div><div class="author-info__name">Mr.wu</div><div class="author-info__description">分享知识</div></div><div class="card-info-data site-data is-center"><a href="/archives/"><div class="headline">文章</div><div class="length-num">63</div></a><a href="/tags/"><div class="headline">标签</div><div class="length-num">32</div></a><a href="/categories/"><div class="headline">分类</div><div class="length-num">25</div></a></div><a id="card-info-btn" target="_blank" rel="noopener" href="https://github.com/xxxxxx"><i class="fab fa-github"></i><span>Follow Me</span></a><div class="card-info-social-icons is-center"><a class="social-icon" href="https://github.com/xxxxxx" target="_blank" title="Github"><i class="fab fa-github"></i></a><a class="social-icon" href="https://blog.csdn.net/blue_gym" target="_blank" title="CSDN"><i class="fa fa-book-open"></i></a><a class="social-icon" href="mailto:shouwangyinying@163.com" target="_blank" title="Email"><i class="fas fa-envelope-open-text"></i></a></div></div><div class="card-widget card-announcement"><div class="item-headline"><i class="fas fa-bullhorn fa-shake"></i><span>公告</span></div><div class="announcement_content">Nice to meet you. please get website connection by Ctrl + F5</div></div><div class="sticky_layout"><div class="card-widget" id="card-toc"><div class="item-headline"><i class="fas fa-stream"></i><span>目录</span><span class="toc-percentage"></span></div><div class="toc-content"><ol class="toc"><li class="toc-item toc-level-1"><a class="toc-link" href="#%E5%8A%A8%E6%80%81%E7%BD%91%E9%A1%B5"><span class="toc-number">1.</span> <span class="toc-text">动态网页</span></a></li><li class="toc-item toc-level-1"><a class="toc-link" href="#%E7%88%AC%E5%8F%96%E6%96%B9%E6%B3%95%E4%B8%80%EF%BC%88selenium%E6%A8%A1%E6%8B%9F%E6%B5%8F%E8%A7%88%E5%99%A8%EF%BC%89"><span class="toc-number">2.</span> <span class="toc-text">爬取方法一（selenium模拟浏览器）</span></a><ol class="toc-child"><li class="toc-item toc-level-2"><a class="toc-link" href="#%E9%85%8D%E7%BD%AE%E7%8E%AF%E5%A2%83"><span class="toc-number">2.1.</span> <span class="toc-text">配置环境</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#%E5%AE%9E%E7%8E%B0%EF%BC%88%E7%88%AC%E5%8F%96%E5%8A%A8%E6%80%81%E9%A1%B5%E9%9D%A2%E5%86%85%E5%AE%B9-%E8%99%9A%E6%8B%9F%E7%82%B9%E5%87%BB%E7%BF%BB%E9%A1%B5%EF%BC%89"><span class="toc-number">2.2.</span> <span class="toc-text">实现（爬取动态页面内容 + 虚拟点击翻页）</span></a><ol class="toc-child"><li class="toc-item toc-level-3"><a class="toc-link" href="#%E8%8E%B7%E5%8F%96%E6%B8%B2%E6%9F%93%E5%90%8E%E7%9A%84%E9%A1%B5%E9%9D%A2"><span class="toc-number">2.2.1.</span> <span class="toc-text">获取渲染后的页面</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E5%AE%9A%E4%BD%8D%E7%88%AC%E5%8F%96%E5%86%85%E5%AE%B9%E7%9A%84%E6%96%B9%E6%B3%95"><span class="toc-number">2.2.2.</span> <span class="toc-text">定位爬取内容的方法</span></a><ol class="toc-child"><li class="toc-item toc-level-4"><a class="toc-link" href="#F12%E6%89%93%E5%BC%80%E9%A1%B5%E9%9D%A2%E5%85%83%E7%B4%A0%E5%A6%82%E5%9B%BE%E8%8E%B7%E5%8F%96xpath"><span class="toc-number">2.2.2.1.</span> <span class="toc-text">F12打开页面元素如图获取xpath</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#%E8%8E%B7%E5%8F%96%E5%85%83%E7%B4%A0%E4%B8%AD%E6%83%B3%E8%A6%81%E7%9A%84%E9%83%A8%E5%88%86"><span class="toc-number">2.2.2.2.</span> <span class="toc-text">获取元素中想要的部分</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#%E8%BF%9B%E4%B8%80%E6%AD%A5%E8%8E%B7%E5%8F%96"><span class="toc-number">2.2.2.3.</span> <span class="toc-text">进一步获取</span></a></li></ol></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E5%AE%8C%E6%95%B4%E4%BB%A3%E7%A0%81"><span class="toc-number">2.2.3.</span> <span class="toc-text">完整代码</span></a></li></ol></li></ol></li><li class="toc-item toc-level-1"><a class="toc-link" href="#%E7%88%AC%E5%8F%96%E6%96%B9%E6%B3%95%E4%BA%8C%EF%BC%88%E9%80%9A%E8%BF%87javsript%E9%80%86%E5%90%91%E8%8E%B7%E5%8F%96%E6%95%B0%E6%8D%AE%E6%8E%A5%E5%8F%A3%EF%BC%89"><span class="toc-number">3.</span> <span class="toc-text">爬取方法二（通过javsript逆向获取数据接口）</span></a><ol class="toc-child"><li class="toc-item toc-level-2"><a class="toc-link" href="#%E7%88%AC%E5%8F%96%E4%BB%8A%E6%97%A5%E5%A4%B4%E6%9D%A1%E7%9A%84%E5%A4%B4%E6%9D%A1%E7%83%AD%E6%A6%9C%EF%BC%88https-www-toutiao-com-%EF%BC%89"><span class="toc-number">3.1.</span> <span class="toc-text">爬取今日头条的头条热榜（https:&#x2F;&#x2F;www.toutiao.com&#x2F;）</span></a><ol class="toc-child"><li class="toc-item toc-level-3"><a class="toc-link" href="#%E5%88%86%E6%9E%90"><span class="toc-number">3.1.1.</span> <span class="toc-text">分析</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E5%AE%9E%E7%8E%B0"><span class="toc-number">3.1.2.</span> <span class="toc-text">实现</span></a></li></ol></li><li class="toc-item toc-level-2"><a class="toc-link" href="#%E7%88%AC%E5%8F%96%E9%A9%AC%E8%9C%82%E7%AA%9D%E7%BD%91%E8%87%AA%E7%94%B1%E8%A1%8C%E5%88%97%E8%A1%A8%EF%BC%88https-www-mafengwo-cn-sales-0-0-M10030-0-0-0-0-0-html%EF%BC%89"><span class="toc-number">3.2.</span> <span class="toc-text">爬取马蜂窝网自由行列表（https:&#x2F;&#x2F;www.mafengwo.cn&#x2F;sales&#x2F;0-0-M10030-0-0-0-0-0.html）</span></a><ol class="toc-child"><li class="toc-item toc-level-3"><a class="toc-link" href="#%E5%88%86%E6%9E%90-1"><span class="toc-number">3.2.1.</span> <span class="toc-text">分析</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E5%AE%9E%E7%8E%B0-1"><span class="toc-number">3.2.2.</span> <span class="toc-text">实现</span></a></li></ol></li></ol></li></ol></div></div><div class="card-widget card-recent-post"><div class="item-headline"><i class="fas fa-history"></i><span>最新文章</span></div><div class="aside-list"><div class="aside-list-item no-cover"><div class="content"><a class="title" href="/pages/%E7%AE%97%E6%B3%95/%E6%99%AE%E9%80%9A%E6%95%B0%E7%BB%84/%E7%BC%BA%E5%A4%B1%E7%9A%84%E7%AC%AC%E4%B8%80%E4%B8%AA%E6%AD%A3%E6%95%B0.html" title="缺失的第一个正数">缺失的第一个正数</a><time datetime="2024-01-06T13:48:58.072Z" title="发表于 2024-01-06 21:48:58">2024-01-06</time></div></div><div class="aside-list-item no-cover"><div class="content"><a class="title" href="/pages/%E7%AE%97%E6%B3%95/%E7%9F%A9%E9%98%B5/%E7%9F%A9%E9%98%B5%E7%BD%AE%E9%9B%B6.html" title="矩阵置零">矩阵置零</a><time datetime="2024-01-06T13:48:58.070Z" title="发表于 2024-01-06 21:48:58">2024-01-06</time></div></div><div class="aside-list-item no-cover"><div class="content"><a class="title" href="/pages/RPC%E6%A1%86%E6%9E%B6/%E6%9C%8D%E5%8A%A1%E8%BF%9C%E7%A8%8B%E8%B0%83%E7%94%A8(RestTemplate%E3%80%81Feign%E3%80%81Dubbo).html" title="服务远程调用(RestTemplate、Feign、Dubbo)">服务远程调用(RestTemplate、Feign、Dubbo)</a><time datetime="2023-12-30T02:24:00.000Z" title="发表于 2023-12-30 10:24:00">2023-12-30</time></div></div><div class="aside-list-item no-cover"><div class="content"><a class="title" href="/pages/%E8%BD%AF%E4%BB%B6%E6%9E%B6%E6%9E%84/%E8%BD%AF%E4%BB%B6%E6%9E%B6%E6%9E%84%E5%A4%8D%E4%B9%A0.html" title="软件架构复习">软件架构复习</a><time datetime="2023-12-23T15:50:20.000Z" title="发表于 2023-12-23 23:50:20">2023-12-23</time></div></div><div class="aside-list-item no-cover"><div class="content"><a class="title" href="/pages/%E7%AE%97%E6%B3%95/%E4%BA%8C%E5%88%86%E6%9F%A5%E6%89%BE/%E6%90%9C%E7%B4%A2%E4%BA%8C%E7%BB%B4%E7%9F%A9%E9%98%B5.html" title="搜索二维矩阵 - 算法 - 二分查找">搜索二维矩阵 - 算法 - 二分查找</a><time datetime="2023-12-04T04:50:20.000Z" title="发表于 2023-12-04 12:50:20">2023-12-04</time></div></div></div></div></div></div></main><footer id="footer" style="background-image: url('https://api.aqcoder.cn/random')"><div id="footer-wrap"><div class="copyright">&copy;2023 - 2024 By Mr.wu</div><div class="framework-info"><span>框架 </span><a target="_blank" rel="noopener" href="https://hexo.io">Hexo</a><span class="footer-separator">|</span><span>主题 </span><a target="_blank" rel="noopener" href="https://github.com/jerryc127/hexo-theme-butterfly">Butterfly</a></div></div></footer></div><div id="rightside"><div id="rightside-config-hide"><button id="readmode" type="button" title="阅读模式"><i class="fas fa-book-open"></i></button><button id="darkmode" type="button" title="浅色和深色模式转换"><i class="fas fa-adjust"></i></button><button id="hide-aside-btn" type="button" title="单栏和双栏切换"><i class="fas fa-arrows-alt-h"></i></button></div><div id="rightside-config-show"><button id="rightside_config" type="button" title="设置"><i class="fas fa-cog fa-spin"></i></button><button class="close" id="mobile-toc-button" type="button" title="目录"><i class="fas fa-list-ul"></i></button><button id="go-up" type="button" title="回到顶部"><span class="scroll-percent"></span><i class="fas fa-arrow-up"></i></button></div></div><div><script src="/js/utils.js"></script><script src="/js/main.js"></script><script src="https://cdn.jsdelivr.net/npm/@fancyapps/ui/dist/fancybox/fancybox.umd.min.js"></script><script src="https://cdn.jsdelivr.net/npm/vanilla-lazyload/dist/lazyload.iife.min.js"></script><div class="js-pjax"></div><canvas id="universe"></canvas>'<script src="/js/universe.js"></script><script src="https://cdn.jsdelivr.net/npm/butterfly-extsrc/dist/activate-power-mode.min.js"></script><script>POWERMODE.colorful = true;
POWERMODE.shake = true;
POWERMODE.mobile = false;
document.body.addEventListener('input', POWERMODE);
</script><script async data-pjax src="//busuanzi.ibruce.info/busuanzi/2.3/busuanzi.pure.mini.js"></script><div id="local-search"><div class="search-dialog"><nav class="search-nav"><span class="search-dialog-title">搜索</span><span id="loading-status"></span><button class="search-close-button"><i class="fas fa-times"></i></button></nav><div class="is-center" id="loading-database"><i class="fas fa-spinner fa-pulse"></i><span>  数据库加载中</span></div><div class="search-wrap"><div id="local-search-input"><div class="local-search-box"><input class="local-search-box--input" placeholder="搜索文章" type="text"/></div></div><hr/><div class="no-result" id="local-search-results"></div><div id="local-search-stats-wrap"></div></div></div><div id="search-mask"></div><script src="/js/search/local-search.js"></script></div></div><script async>window.onload=function(){var a=document.createElement('script'),b=document.getElementsByTagName('script')[0];a.type='text/javascript',a.async=!0,a.src='/sw-register.js?v='+Date.now(),b.parentNode.insertBefore(a,b)};</script></body></html>